{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Seq2Seq Translation using skorch\n",
    "\n",
    "This notebook demonstrates how to use the code from the [sequence-to-sequence tutorial][1] with skorch!\n",
    "\n",
    "[1]: http://pytorch.org/tutorials/intermediate/seq2seq_translation_tutorial.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import sklearn\n",
    "import skorch\n",
    "\n",
    "from skorch.utils import params_for\n",
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading data\n",
    "\n",
    "We use the same data loading routines as the tutorial but for the\n",
    "sake of brevity we will leave it out of the notebook and simply\n",
    "import the code from `data.py`. If you don't have the data already, \n",
    "you can download it from http://www.manythings.org/anki/. \n",
    "\n",
    "For this notebook to run you need to\n",
    "\n",
    "- create a `data/` dir\n",
    "- download http://www.manythings.org/anki/fra-eng.zip\n",
    "- extract `fra.txt` from `fra-eng.zip` to `data/eng-fra.txt`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading lines...\n",
      "Read 135842 sentence pairs\n",
      "Trimmed to 10853 sentence pairs\n",
      "Counting words...\n",
      "Counted words:\n",
      "fra 4489\n",
      "eng 2925\n"
     ]
    }
   ],
   "source": [
    "input_lang, output_lang, pairs = data.prepareData('eng', 'fra', True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['je suis ravie .', 'i m thrilled .']\n"
     ]
    }
   ],
   "source": [
    "print(random.choice(pairs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model\n",
    "\n",
    "The seq2seq model code itself consists of the encoder and the decoder which we can use 1:1 from the example."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EncoderRNN(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, n_layers=1):\n",
    "        super(EncoderRNN, self).__init__()\n",
    "        self.n_layers = n_layers\n",
    "        self.hidden_size = hidden_size\n",
    "\n",
    "        self.embedding = nn.Embedding(input_size, hidden_size)\n",
    "        self.gru = nn.GRU(hidden_size, hidden_size)\n",
    "\n",
    "    def forward(self, input, hidden):\n",
    "        embedded = self.embedding(input).view(1, 1, -1)        \n",
    "        output = embedded\n",
    "        for i in range(self.n_layers):\n",
    "            output, hidden = self.gru(output, hidden)\n",
    "        return output, hidden\n",
    "\n",
    "    def initHidden(self):\n",
    "        return torch.zeros(1, 1, self.hidden_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AttnDecoderRNN(nn.Module):\n",
    "    def __init__(self, hidden_size, output_size, n_layers=1, dropout_p=0.1, max_length=data.MAX_LENGTH):\n",
    "        super(AttnDecoderRNN, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.output_size = output_size\n",
    "        self.n_layers = n_layers\n",
    "        self.dropout_p = dropout_p\n",
    "        self.max_length = max_length\n",
    "\n",
    "        self.embedding = nn.Embedding(self.output_size, self.hidden_size)\n",
    "        self.attn = nn.Linear(self.hidden_size * 2, self.max_length)\n",
    "        self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)\n",
    "        self.dropout = nn.Dropout(self.dropout_p)\n",
    "        self.gru = nn.GRU(self.hidden_size, self.hidden_size)\n",
    "        self.out = nn.Linear(self.hidden_size, self.output_size)\n",
    "\n",
    "    def forward(self, input, hidden, encoder_output, encoder_outputs):\n",
    "        embedded = self.embedding(input).view(1, 1, -1)\n",
    "        embedded = self.dropout(embedded)\n",
    "\n",
    "        attn_weights = F.softmax(\n",
    "            self.attn(torch.cat((embedded[0], hidden[0]), 1)), dim=-1)\n",
    "        attn_applied = torch.bmm(attn_weights.unsqueeze(0),\n",
    "                                 encoder_outputs.unsqueeze(0))\n",
    "\n",
    "        output = torch.cat((embedded[0], attn_applied[0]), 1)\n",
    "        output = self.attn_combine(output).unsqueeze(0)\n",
    "\n",
    "        for i in range(self.n_layers):\n",
    "            output = F.relu(output)\n",
    "            output, hidden = self.gru(output, hidden)\n",
    "\n",
    "        output = F.log_softmax(self.out(output[0]), dim=-1)\n",
    "        return output, hidden, attn_weights\n",
    "\n",
    "    def initHidden(self):\n",
    "        return torch.zeros(1, 1, self.hidden_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Seq2Seq Glue Model\n",
    "\n",
    "In the original code the encoder and decoder are combined in the training and\n",
    "evaluation code. We refrain from doing this and define a `Seq2Seq`-model that\n",
    "unites both and allows us to have a single point of entry and avoids code\n",
    "duplication for inference and training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Seq2Seq(nn.Module):\n",
    "    def __init__(\n",
    "        self, \n",
    "        encoder, \n",
    "        decoder,\n",
    "        teacher_forcing_ratio=0.5,\n",
    "        hidden_size=256,\n",
    "        max_length=data.MAX_LENGTH,\n",
    "        **kwargs\n",
    "    ):\n",
    "        super().__init__()\n",
    "        self.encoder = encoder(hidden_size=hidden_size, **params_for('encoder', kwargs))\n",
    "        self.decoder = decoder(hidden_size=hidden_size, **params_for('decoder', kwargs))\n",
    "        self.max_length = max_length\n",
    "        self.teacher_forcing_ratio = teacher_forcing_ratio\n",
    "        \n",
    "    def forward(self, x, y=None):\n",
    "        # Encode the input x to the thought vector stored in encoder_outputs.\n",
    "        encoder_hidden = self.encoder.initHidden().to(x.device)\n",
    "        encoder_outputs = torch.zeros(self.max_length, self.encoder.hidden_size).to(x.device)\n",
    "                \n",
    "        for ei in range(x.size(1)):\n",
    "            encoder_output, encoder_hidden = self.encoder(x[0, ei], encoder_hidden)\n",
    "            encoder_outputs[ei] = encoder_output[0][0]\n",
    "\n",
    "        use_teacher_forcing = y is not None and random.random() < self.teacher_forcing_ratio\n",
    "        ys = []\n",
    "                \n",
    "        target_length = self.max_length if y is None else y.size(1)\n",
    "        target_variable = y\n",
    "\n",
    "        # Decode the thought vector into the target sequence beginning with a \n",
    "        # start-of-sentence (SOS) token.\n",
    "        decoder_input = torch.LongTensor([[data.SOS_token]]).to(x)\n",
    "        decoder_hidden = encoder_hidden\n",
    "\n",
    "        if use_teacher_forcing:\n",
    "            # Teacher forcing: Feed the target as the next input\n",
    "            for di in range(target_length):\n",
    "                decoder_output, decoder_hidden, decoder_attention = self.decoder(\n",
    "                    decoder_input, decoder_hidden, encoder_output, encoder_outputs)\n",
    "                ys.append(decoder_output)\n",
    "                decoder_input = target_variable[0, di]  # Teacher forcing\n",
    "\n",
    "        else:\n",
    "            # Without teacher forcing: use its own predictions as the next input\n",
    "            for di in range(target_length):\n",
    "                decoder_output, decoder_hidden, decoder_attention = self.decoder(\n",
    "                    decoder_input, decoder_hidden, encoder_output, encoder_outputs)\n",
    "                topv, topi = decoder_output.data.topk(1)\n",
    "                ni = topi[0][0]\n",
    "\n",
    "                decoder_input = torch.LongTensor([[ni]]).to(x)\n",
    "\n",
    "                ys.append(decoder_output)\n",
    "                if ni == data.EOS_token:\n",
    "                    break\n",
    "\n",
    "        return torch.stack(ys, dim=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training\n",
    "\n",
    "After defining the necessary components we can prepare everything for the training itself."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prerequisites\n",
    "\n",
    "Some fluff taken 1:1 from the original code."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def indexesFromSentence(lang, sentence):\n",
    "    return [lang.word2index[word] for word in sentence.split(' ')]\n",
    "\n",
    "\n",
    "def variableFromSentence(lang, sentence):\n",
    "    indexes = indexesFromSentence(lang, sentence)\n",
    "    indexes.append(data.EOS_token)\n",
    "    result = Variable(torch.LongTensor(indexes).view(-1, 1))\n",
    "    return result.to('cuda' if use_cuda else 'cpu')\n",
    "    \n",
    "\n",
    "def variablesFromPair(pair):\n",
    "    input_variable = variableFromSentence(input_lang, pair[0])\n",
    "    target_variable = variableFromSentence(output_lang, pair[1])\n",
    "    return (input_variable, target_variable)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Wrapping the seq2seq model in skorch\n",
    "\n",
    "Now we can wrap our model into a `skorch.NeuralNet` where we \n",
    "define how our seq2seq model is trained. For this we need only a few things:\n",
    "\n",
    "1. A proper loss\n",
    "2. The training step (data flow in and out of the model)\n",
    "\n",
    "Since we are dealing with variable length sequences produced\n",
    "by the model we also have to deal with those in the evaluation step,\n",
    "namely `predict` and `predict_proba` where we can solve this issue \n",
    "by returning a sequence that is padded to a fixed length.\n",
    "\n",
    "Another extra is that the tutorial uses two separate optimizers for\n",
    "the encoder and the decoder. We can implement this as well, even though\n",
    "it does not make a difference with SGD, it might make a difference when\n",
    "using Adam, for example. We can evaluate this difference by doing a \n",
    "grid search over the possible optimizer encoders. For this we will\n",
    "need a `score()` function as well."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Trainer(skorch.NeuralNet):\n",
    "    \n",
    "    def __init__(\n",
    "        self, \n",
    "        *args, \n",
    "        optimizer_encoder=torch.optim.SGD, \n",
    "        optimizer_decoder=torch.optim.SGD,\n",
    "        **kwargs\n",
    "    ):\n",
    "        self.optimizer_encoder = optimizer_encoder\n",
    "        self.optimizer_decoder = optimizer_decoder\n",
    "        super().__init__(*args, **kwargs)\n",
    "    \n",
    "    def initialize_optimizer(self):\n",
    "        kwargs = self._get_params_for('optimizer_encoder')\n",
    "        self.optimizer_encoder_ = self.optimizer_encoder(self.module_.encoder.parameters(), **kwargs)\n",
    "        kwargs = self._get_params_for('optimizer_decoder')\n",
    "        self.optimizer_decoder_ = self.optimizer_decoder(self.module_.decoder.parameters(), **kwargs)\n",
    "\n",
    "    def train_step(self, Xi, yi):\n",
    "        self.module_.train()\n",
    "        \n",
    "        self.optimizer_encoder_.zero_grad()\n",
    "        self.optimizer_decoder_.zero_grad()\n",
    "\n",
    "        y_pred = self.infer(Xi, yi)\n",
    "        loss = self.get_loss(y_pred, yi, X=Xi, training=True)\n",
    "        loss.backward()\n",
    "        \n",
    "        self.optimizer_encoder_.step()\n",
    "        self.optimizer_decoder_.step()\n",
    "        \n",
    "        return {'loss': loss, 'y_pred': y_pred}\n",
    "\n",
    "    def infer(self, Xi, yi=None):\n",
    "        Xi = skorch.utils.to_tensor(Xi, device=self.device)\n",
    "        yi = skorch.utils.to_tensor(yi, device=self.device) if yi is not None else None\n",
    "        return self.module_(Xi, yi)\n",
    "    \n",
    "    def get_loss(self, y_pred, y_true, **kwargs):\n",
    "        y_true = y_true[:, :y_pred.size(1)]        \n",
    "        y_pred_flat = y_pred.view(y_pred.size(0) * y_pred.size(1), -1)\n",
    "        y_true_flat = y_true.view(y_true.size(0) * y_true.size(1))\n",
    "        \n",
    "        return super().get_loss(\n",
    "            y_pred_flat,\n",
    "            y_true_flat,\n",
    "            **kwargs)\n",
    "    \n",
    "    def _predict(self, X, most_probable=True):\n",
    "        # return either predicted word probabilities or the most probable \n",
    "        # word using argmax.\n",
    "        y_probas = []\n",
    "        for yp in self.forward_iter(X, training=False):\n",
    "            if most_probable:\n",
    "                pad = np.zeros((yp.size(0), data.MAX_LENGTH))\n",
    "                pad[:, :yp.size(1)] = skorch.utils.to_numpy(yp.max(-1)[-1])\n",
    "            else:\n",
    "                pad = np.zeros((yp.size(0), data.MAX_LENGTH, yp.size(-1)))\n",
    "                pad[:, :yp.size(1)] = skorch.utils.to_numpy(yp)\n",
    "            y_probas.append(pad)\n",
    "        y_proba = np.concatenate(y_probas, 0)\n",
    "        return y_proba\n",
    "    \n",
    "    def predict_proba(self, X):\n",
    "        return self._predict(X, most_probable=False)\n",
    "    \n",
    "    def predict(self, X):\n",
    "        return self._predict(X, most_probable=True)\n",
    "    \n",
    "    def score(self, X, y):\n",
    "        y_pred = self.predict(X)\n",
    "        y_true = y_pred.copy()\n",
    "        \n",
    "        for i, yi in enumerate(y):\n",
    "            yi = skorch.utils.to_numpy(yi.squeeze())\n",
    "            y_true[:, :len(yi)] = yi\n",
    "        \n",
    "        return sklearn.metrics.accuracy_score(y_true.flatten(), y_pred.flatten())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Running the training process"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For reasons of practicality we set `n_iters` to a relatively low number so that the training\n",
    "finishes in a few minutes rather than hours. If you want better accuracy, you just need\n",
    "to increase this number. The original code uses `n_iters=75000`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_iters = 7500\n",
    "use_cuda = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We create the training pairs from the first `n_iters` samples. Since the training data is sorted\n",
    "by length this has the added benefit that we can evaluate shorter (and therefore easier) sequences\n",
    "fast and increase the difficulty incrementally by increasing `n_iters`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_pairs = [variablesFromPair(pairs[i]) for i in range(n_iters)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can now instantiate the `Trainer` with the parameters we want to train our model with:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = Trainer(\n",
    "    criterion=torch.nn.NLLLoss,\n",
    "    \n",
    "    # We extended the trainer to support two optimizers\n",
    "    # but to get the behavior of one optimizer we can \n",
    "    # simply use SGD for both, just like in the tutorial.\n",
    "    optimizer_encoder=torch.optim.SGD,\n",
    "    optimizer_encoder__lr=0.01,\n",
    "    optimizer_decoder=torch.optim.SGD,\n",
    "    optimizer_decoder__lr=0.01,\n",
    "    \n",
    "    module=Seq2Seq,\n",
    "    module__hidden_size=256,\n",
    "\n",
    "    module__encoder=EncoderRNN,\n",
    "    module__encoder__input_size=input_lang.n_words,\n",
    "    \n",
    "    module__decoder=AttnDecoderRNN,\n",
    "    module__decoder__output_size=output_lang.n_words,\n",
    "    module__decoder__dropout_p=0.1,\n",
    "    \n",
    "    # We have no internal validation.\n",
    "    train_split=None,\n",
    "    \n",
    "    # The decoding code is not meant to be batched\n",
    "    # so we have to deal with a batch size of 1 for\n",
    "    # both training and validation/prediction.\n",
    "    iterator_train__batch_size=1,\n",
    "    iterator_valid__batch_size=1,\n",
    "    \n",
    "    # We are training only one large epoch.\n",
    "    max_epochs=1,\n",
    "    \n",
    "    # Training takes a long time, add a progress bar\n",
    "    # to see how far in we are. Since we are doing \n",
    "    # grid search with cross-validation splits and the\n",
    "    # total amount of batches_per_epoch varies, we set\n",
    "    # the batches_per_epoch method to 'auto', instead \n",
    "    # of the default, 'count'.\n",
    "    callbacks=[\n",
    "        skorch.callbacks.ProgressBar(batches_per_epoch='auto'),\n",
    "    ],\n",
    "    \n",
    "    device=('cuda' if use_cuda else 'cpu'),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we just create our X/y pairs, define our parameters for the grid search and run `.fit()` to find the best parametrization of our `Trainer` instance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array([n[0].data for n in training_pairs], dtype=object)\n",
    "y = np.array([n[1].data for n in training_pairs], dtype=object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use a grid search to determine whether it makes sense to use different optimizers for the encoder and the decoder, in this case we test between Adam and SGD. We expect it to run for `3 * 4 = 12` training iterations as we are using the default 3-fold CV split and have 4 parameter combinations:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "    'optimizer_encoder': [torch.optim.SGD, torch.optim.Adam],\n",
    "    'optimizer_decoder': [torch.optim.SGD, torch.optim.Adam],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "gs = GridSearchCV(trainer, params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Automatic pdb calling has been turned ON\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7e75f0415da64b80ae1c97d5f17fe153",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in Jupyter Notebook or JupyterLab, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another notebook frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "  epoch    train_loss       dur\n",
      "-------  ------------  --------\n",
      "      1        \u001b[36m3.2864\u001b[0m  130.2924\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "11fe78af30414f389d3d519d3e5531c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in Jupyter Notebook or JupyterLab, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another notebook frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "  epoch    train_loss       dur\n",
      "-------  ------------  --------\n",
      "      1        \u001b[36m3.0170\u001b[0m  123.9097\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0f7b02462e2c4489a73a4664bdd35c9d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in Jupyter Notebook or JupyterLab, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another notebook frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "  epoch    train_loss       dur\n",
      "-------  ------------  --------\n",
      "      1        \u001b[36m2.7005\u001b[0m  116.5283\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "183df28b6496402db814e72682fc28e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in Jupyter Notebook or JupyterLab, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another notebook frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "  epoch    train_loss       dur\n",
      "-------  ------------  --------\n",
      "      1        \u001b[36m3.0571\u001b[0m  132.8406\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "15985dd552b64f78a7cdf3cc0d752550",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in Jupyter Notebook or JupyterLab, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another notebook frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "  epoch    train_loss       dur\n",
      "-------  ------------  --------\n",
      "      1        \u001b[36m2.8227\u001b[0m  126.8780\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99c7c74c002d47e7b3229331cca36d05",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in Jupyter Notebook or JupyterLab, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another notebook frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "  epoch    train_loss       dur\n",
      "-------  ------------  --------\n",
      "      1        \u001b[36m2.5147\u001b[0m  120.3536\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "522a2ced58ac4ba9a5d825cfb11ab1bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in Jupyter Notebook or JupyterLab, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another notebook frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "  epoch    train_loss       dur\n",
      "-------  ------------  --------\n",
      "      1        \u001b[36m7.3987\u001b[0m  134.3116\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1cbe04b8a221453b9ca1f713718bd41b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in Jupyter Notebook or JupyterLab, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another notebook frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "  epoch    train_loss       dur\n",
      "-------  ------------  --------\n",
      "      1        \u001b[36m8.0080\u001b[0m  131.6806\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "79363085cf4242f1bd634633c0a1e5bc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in Jupyter Notebook or JupyterLab, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another notebook frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "  epoch    train_loss       dur\n",
      "-------  ------------  --------\n",
      "      1        \u001b[36m7.5580\u001b[0m  126.8411\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7980a02050634d23b55a664ab5ccad18",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in Jupyter Notebook or JupyterLab, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another notebook frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "  epoch    train_loss       dur\n",
      "-------  ------------  --------\n",
      "      1       \u001b[36m12.5318\u001b[0m  154.2361\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a0971665a25a4253a8b84fea124af8ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in Jupyter Notebook or JupyterLab, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another notebook frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "  epoch    train_loss       dur\n",
      "-------  ------------  --------\n",
      "      1       \u001b[36m11.7451\u001b[0m  142.4930\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "72412052169a471aa6c1c5a5cf8f7654",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in Jupyter Notebook or JupyterLab, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another notebook frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "  epoch    train_loss       dur\n",
      "-------  ------------  --------\n",
      "      1       \u001b[36m11.4114\u001b[0m  132.3407\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33857e7df1c2450f8ee18285540204d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in Jupyter Notebook or JupyterLab, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another notebook frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "  epoch    train_loss       dur\n",
      "-------  ------------  --------\n",
      "      1        \u001b[36m2.8116\u001b[0m  199.1263\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=None, error_score='raise',\n",
       "       estimator=<class '__main__.Trainer'>[uninitialized](\n",
       "  module=<class '__main__.Seq2Seq'>,\n",
       "  module__decoder=<class '__main__.AttnDecoderRNN'>,\n",
       "  module__decoder__dropout_p=0.1,\n",
       "  module__decoder__output_size=2925,\n",
       "  module__encoder=<class '__main__.EncoderRNN'>,\n",
       "  module__encoder__input_size=4489,\n",
       "  module__hidden_size=256,\n",
       "),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'optimizer_encoder': [<class 'torch.optim.sgd.SGD'>, <class 'torch.optim.adam.Adam'>], 'optimizer_decoder': [<class 'torch.optim.sgd.SGD'>, <class 'torch.optim.adam.Adam'>]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring=None, verbose=0)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%pdb on\n",
    "gs.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So let's look at the best parameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'optimizer_decoder': torch.optim.sgd.SGD,\n",
       " 'optimizer_encoder': torch.optim.sgd.SGD}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And there we have it: Using two separate optimizers is irrelevant for the set of parameters we evaluated. The best is to just use SGD in for both the encoder and decoder, or, as SGD is stateless, just use a single SGD for both."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can now use the estimator with the best configuration to observe how well it translates, but first, let's save the model for later use:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "ef = gs.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "ef.save_params(f_params='model.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot Training Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f54ae99f668>]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJztnXmcHGWd/z9P39fcR+5kEhKScCcE\nCESQI5wqqIsHriiwu7iCCq6/xYDHuqKLu6usq64iyqGIgsuhLvd9CSYkIYRgEkJCyJ25Z/o+n98f\nVU/1U9VV3dUz3ZNq5vt+vfJKT3dN9TPdVZ/61vdknHMQBEEQjYPrUC+AIAiCqA4SboIgiAaDhJsg\nCKLBIOEmCIJoMEi4CYIgGgwSboIgiAaDhJsgCKLBIOEmCIJoMEi4CYIgGgxPPXba2dnJe3p66rFr\ngiCI9yTr1q3r55x32dm2LsLd09ODtWvX1mPXBEEQ70kYY+/a3ZZcJQRBEA0GCTdBEESDQcJNEATR\nYJBwEwRBNBgk3ARBEA0GCTdBEESDQcJNEATRYDhKuH/09DY8/1bfoV4GQRCEo3GUcN/y/Ha8tI2E\nmyAIohyOEm6PiyGbp+HFBEEQ5XCUcPs8LmTzhUO9DIIgCEfjKOH2uEi4CYIgKuEo4fZ6GHLkKiEI\ngiiLs4Tb5UKGLG6CIIiyOEu43S6yuAmCICpgS7gZY62MsfsYY1sYY5sZYyfXYzEeNyMfN0EQRAXs\nDlL4bwCPcc4vZoz5AITqsRiv24VsgSxugiCIclQUbsZYM4DTAFwGAJzzDIBMPRbjdTNkc2RxEwRB\nlMOOq2QegD4AdzDGXmOM/ZIxFq7HYrxuF3IFEm6CIIhy2BFuD4ClAH7GOV8CIA5glXEjxtiVjLG1\njLG1fX1jK1v3uF3IUHCSIAiiLHaEew+APZzz1erP90ERch2c81s558s458u6umwNKi7B52bIUXCS\nIAiiLBWFm3N+AMBuxthC9amzAPy1HouhykmCIIjK2M0q+SKAu9WMkh0ALq/HYrweyuMmCIKohC3h\n5pxvALCszmuB18WocpIgCKICVDlJEATRYDhKuKlykiAIojKOEm6vm4KTBEEQlXCUcCuDFMhVQhAE\nUQ5HCbfHxahykiAIogKOEm7FVcLBOVndBEEQVjhMuBkAIEcdAgmCICxxmHAry6EAJUEQhDWOEm6P\nJtxkcRMEQVjhKOH2qa4SsrgJgiCscZRwC4ubqicJgiCscZRwk4+bIAiiMg4TbnKVEARBVMJhwk3B\nSYIgiEo4Srg9LrK4CYIgKuEo4fZ6yMdNEARRCWcJt0vNKqHKSYIgCEucJdwiOJkji5sgCMIKRwm3\nVjlJFjdBEIQljhJunxBusrgJgiAscZRwe7TugCTcBEEQVjhKuEUed4byuAmCICxxmHCrFjelAxIE\nQVjiMOGmPG6CIIhKOEq4PVqvEnKVEARBWOEo4faRxU0QBFERRwk39eMmCIKojKOEWwQnM2RxEwRB\nWOIs4XaRxU0QBFEJRwm3y8XgdjHycRMEQZTBUcINKD25s1Q5SRAEYYnHzkaMsZ0AogDyAHKc82X1\nWpDP7UI2R64SgiAIK2wJt8oZnPP+uq1ExeNm1KuEIAiiDI5zlXjdLvJxEwRBlMGucHMATzDG1jHG\nrqznghThJlcJQRCEFXZdJSs45/sYY90AnmSMbeGcvyBvoAr6lQAwe/bsMS/I66asEoIgiHLYsrg5\n5/vU/3sBPAjgRJNtbuWcL+OcL+vq6hrzgrxuF+VxEwRBlKGicDPGwoyxJvEYwDkANtVrQR63iyon\nCYIgymDHVTIFwIOMMbH9bznnj9VrQT43o37cBEEQZago3JzzHQCOnYC1AFAsbgpOEgRBWOPAdEAK\nThIEQZTDgcJNedwEQRDlcKRw5wrkKiEIgrDCccId9LoRS+UO9TIIgiAci+OEe0pzAAdHU4d6GQRB\nEI7FgcLtRzyTRyxNVjdBEIQZjhPu7mY/AJDVTRAEYYHjhLsjrAj3YDxziFdCEAThTBwn3GG/UhMU\nJ1cJQRCEKQ4UbjcAIJHJH+KVEARBOBPnCbePLG6CIIhyOE+4VVcJWdwEQRDmOE64Qz7FVRLPkMVN\nEARhhuOE2+9xwe1iSKTJ4iYIgjDDccLNGEPI56YCHIIgCAscJ9yAEqBMkKuEIAjCFEcKd8jvRpyC\nkwRBEKY4UrjDPg8S5CohCIIwxZHCHfKRxU0QBGGFI4U77CcfN0EQhBXOFW5KByQIgjDFkcId8rqp\nAIcgCMICRwq33+tCOkcDgwmCIMxwpnB7XEhnSbgJgiDMcKRwB7xupHN5cE7T3gmCIIw4Urj9HhcK\nHMjm9cK9ayCBu1e/e4hWRRAE4Qw8h3oBZvg9SofAdC4Pn6d4bbn09tV4dyCBi46bgYjfkUsnCIKo\nO860uL3KsowByv3DygDhR9/YP+FrIgiCcAqOFO6AZnHrhbunMwQAeHPf6ISviSAIwik4Urg1izur\nL8JpC/kAAF1N/glfE0EQhFNwpnCrfu2UISVwMJ4BUCroBEEQkwmHCncxOCnI5QvYORBXn6ccb4Ig\nJi+2hZsx5maMvcYYe6ieCwKKFrcs0H2xtJYeSMJNEMRkphqL+xoAm+u1EBm/tzQ4OZzIao9lS5wg\nCGKyYUu4GWMzAXwAwC/ruxyFoo+7KNAjSUm4qRyeIIhJjF2L+4cArgMwIYoZMMnjHohltMfkKiEI\nYjJTUbgZYx8E0Ms5X1dhuysZY2sZY2v7+vrGtSgtOClZ3H1RpfhmektAZ4kTBEFMNuxY3CsAXMgY\n2wngHgBnMsZ+Y9yIc34r53wZ53xZV1fXuBZlVjnZG03D42KY0hIgi5sgiElNReHmnF/POZ/JOe8B\n8EkAz3DOP13PRflNKif7oml0RvwIqp0DCYIgJisOzeMuDU72xdLoavIrvbrJ4iYIYhJTVYs9zvlz\nAJ6ry0okzPK4hxJZtIa88HvclFVCEMSkxpEWN2NMtayLFncinUPE71HHmpGrhCCIyYtjm1qL8WUP\nrN+Dbb0xxNM5hP0euBilAxIEMblxrnCrQch/+v3rAICWoBdhnxscoHRAgiAmNc4VbsPA4EQmh6DP\ng3yhQBY3QRCTGmcLtyTQ2TxH0OtGNs9IuAmCmNQ4MjgJFCe9ywR9Lvg9LuQLHLk8iTdBEJMTR1vc\nxkEKAbVrIKAEKD1ux153CIIg6oZjhXvfcEqbeCMwCneYJpgRBDEJcaxwHxhNlTwX8LrBuTJMgTJL\nCIKYrDjW1/DRpTNKngt4XAj6lGtNIkPCTRCEOdFUVjPy3os4VrhdjJU8F0vnEPG7tccEQRBG9g0n\ncfS3nsBdf3m34raZXAHZGiU69EZTNdtXJRwr3FOaSx3Yx8xsQcTvBQDESbgJgjDh1Z2DAICHN+6v\nuO3CbzyKc3/4wrjf80+v78OJ330aF/7kz+Pelx0cK9ynHNapPe6M+PD375uL+d1NCKsWdzRFwk0Q\n7yUKBY7hRKbyhgC2HYzipkc3o1AodYdcc88GAEBToHIIj3NgR1+87DZ/eG0vnvzrwbLbfOl3rwEA\nNu8frfietcCxwi1b3Fe8by6+/sEjAABNZHETRMPAOccPntiKXQOJitv+4MmtOO7bT2LfcNJym1y+\ngE/e+grO/q8X8PPnd2DeDY/goEkiAwA0B7xjXrfMtfduwD/8eq2tbRd0R2rynpVwrHBPbw1qj8Vg\nBQCaxU0+boJwPgdGU/jxM2/j8jvXVNz23lf3AAB+U8Y33R/L4C87BnXP3bNmt+5nYWkPVbDeZWvd\nKpApPz8QS1vu65iZLQCAnMkdQD1wrHCHfMXbHPnDi6hfCgk3QTifmOrS7I1aix4AfO6utehXhfG2\nl96x3M4sDXhaS0B7nM0XNDeqsQ7ESFTSkJ0WdwTDiaz2+PjvPIXeqLl1L4aZ23X1jBfHCrfMqOTP\n9nvc8LoZCTdBNABCHCtlWzz+ZtGHfOGx0y23Mzvv948UxVQW2oEKwj2aLG571d3rTbfZ0a/3f3/y\n1r+YbjcYz+BTJ83Gmq+tLPuetcLRwn3EtGYApf7siN+jXckJgijSH0vjmS3lA2kTibB+U9kC8hZu\nBKOboq+MS0Ls72PHz8S2756PGa1BbO+Laa+PJBWx7mryo6+ClT8iCfcMyTUrY/Sfn3JYR8k2yUwe\nyWweM1qD8E5QGw5HC/f3/uZoAMCZi7p1z4f9HgpOEoQJV9z5Kq64c21FN8FEIRtYX//DJtNtRpPK\nNpecOAtnLepG76i14AqL+zMn98DrdmFWexB7pWCmsLgjfg/SuQL+6fcbLPcl3B4+j0sTfCPic1xz\nw1kAgN/8ZVeJu2Qgrqy3I+yzfK9a42jhPmZmK7bceB5WzO/UPR/xe3TuE4IgFERq29k3P3+IV6IQ\nTRWt2t+t2WW6Tb8qfMvndaC72W/pRwaAWFoVZjXWNaU5oLOshRUtApQPrN9rua/fq8HQk+a2Y/eg\neSaL8Fm3hoqivP7dId022w4qFn/IP3EdRBwt3IC+sZSgM+LXrnIEMRl462AUPasexi9f3FF2u5lt\nyi1/Jf/uRGH0SZvlXYvAXkfYj+6mAAbiGUufuLDgI6pItod9ursLIdzfVNOH20Je0xm1nHM89uYB\nAMDS2W04GE0hkSk1BgfjWUT8Hvg81lL5yo4BAMqUronC8cJtRnezv+ztFEE4iQt/8hIuvW31uPbx\n0rZ+AMB3Ht5cNnNh4dQmAErRWrVY+aDHg7gzvuGCRQD0mRwCkWbXHvahu9kPzqFlmBiJpRURFhZ1\nR9iHWDqnibMQ7nldEZx/1FQMJbL44m9fK9lPXOp1dNSMFnAObNpbWjwznMigNaQI8tNfeb/yNyWL\nf8OeoYSWBXNiT7v5h1AHGlO4mwLojabe001kiPcGuwcT2LhnBC+qwjsWdg0k8O2H/qr9/PMXrK3u\npCpI1QbJUtk8jv/Ok7i9TCreWBiIpdEW8qJd7cE8ImV9CPpVi7kz4sOUJiW1z8owi6Wz8LgY/KoF\n3Kb6lYfiyn6FqDYHPHh99zAA4AmTqsch9T3/4+JjcKyag339AxtLthtMZNCmukm6mpS/YVRy/3z5\n3g3aBS/oK/UO1IuGFO4pzX5k8xxDJgdBLeCc4+GN+2nKDjFuHt1U7Jcx1uPp+W19up/LZUuIrpkD\nsUxVhs2+4SSGE1l8+6G/omfVw3jrYNRy26/etxEfu+VlW/uNpnJoDfnQqroRhk2CgEaLGyjN5hAM\nxrNoCXrB1CZ0IiAo3CUjScW14XG7sG/E2lcuLPPWoBfdzcrFYntfvOQzG0pktYtDxOcBY/r0ZLne\nZCJpUOFWPmirL3fT3hE8u7V3zPt/+I39uPq36/GLF2trfRCTD3GsAsDqdwbLbFlKLl/Afz6+Bd+Q\nsjHmdIR0AT8jcdVPm8kXsKaK9zMG+8/5L/PGS5xz3Lt2N17dOWTrwpDI5BHwutEWVoTbzNgaiCnu\nCI/bpX1eVgU7/bG0ZvkC0KxhWbiFr1m4Nhap7iMZkcPdrG576fI56vP6z2EonkGb6ipxuZiSGCGl\nET7/lnJRfeemCyw+gfrQkMItvixjSWsqm8dgPIMP/vglXH7Hq2X38eK2PvSseti0Kcxf1GCDlZ/t\nvcaL2/rwyvaBQ72M9yTySf63v6zOzz3/a4/if57drv183XkLsXBKE97ujVn+jlyAclmFc0DG7GJg\nFkiUc5+TNoaZpLJ5hHxudEYUse01MbYGExm0q1ZtR9gHxsoLt9gXAHSovnyRrDCSzGpifFhXBB9f\nNtO09F1cqISv/KR5in/aOMBlSHKVAEr/k1GTz4qZtKGuJw0p3CHVl2Qsf73yrnVYeuOT2s9WFsHu\nwQQuvU3pnfDle0vzPPujxST+ycClt63BJb8wrwgjrHll+wC298WQyuZxy/PbkcmVukJqmbZ61enz\nMb87gncHEqZZF5xz7BtOaoFJO8KqrTNZuk6zKVS7Boul4Y+8caDifhOZHIJeN6aqZen7TdwXg7GM\n5vLwuF3ojPhNBR5Q3ERmFrfwWY8ms2iWugJ2RPwYjJe6jUTetrDOu4VvXUpFFOXzOuEOenWfVZPf\ng8tO6Sn3EdSFhhRuEQQwTsF54S29L/Ci/zHvjSuL9bKetpLXRYS6MMmCnxPVZ2GiqUe2BABc8ou/\n4KwfPI/bXnoH33t0C+74c6lrbTSZ1QJpwNi7Wj6j3vbPaAsiV+BaCp3MUCKLdK6Ajy+bBQDwuOxb\ngcKK/PLKw3Hn5ScAgK4iUSB37vt///s6dg+W7/qXzBYQ9Lnh97jRFPCYFgYZrdruJr+pxc05Vy3u\n4rZNagdAkXYou0oAxYLP5nnJBVSkS3aoQdN2E1eOuHsRbh5ACXqKu5NkJo9oOndIDLzGFG6vuXAb\n2bhnBD2rHsZR//K47vk9Q8WDz2zSjkgVmmxl9R/6yUt123fvaAr/8sdN+PK9G3DFnfZv4cfKc1t7\nkcrm8cJbfTjshkc091c9uH+dUshx06NbTCw7RUhEqbSZFWuG2E9nxIe3v3s+5nVF1J8VkTBz4wlR\nPXpGCz54zDTMag/Z/huEGP3dqXO1VhPv9Jf2qd47rF//qf/xbNn9JlWLGyjNuRYMSSl3gCLcZvGr\nWDqHVLagc5X4PC54XExLEzQKt3CbrDZ8/wOxDEI+t2YEar5y6XMVhox8UWkKeLFxzwgAYO27Sgwh\naFJrUm8aU7hNXCVmSfYCYxGA7M8za1ojDq7JNqxh92CybEvN8XDDg5vwq1fexYOv7cUzW3pxzLce\nr/xLY2R7XwyX3fEqFn3jMXzmdsUl9gWTXN7xILtF5EZERktxNKX4XK9deTgA4I8b9tnafyqr7P+K\n982FR0rtE6Jl1s9DuCGmtQbRFvJVbGsqE0vlwBgQ8ro1C/LOl3eWvkeZXtlmJFUfN6AI959e34cD\nBnfJaDKnE9spzQFTi/vOPyvrMfqYcwWOW55XYgHDSf1FQORWG/c3EEtr/nGgKPDyOS90QBbupzYf\nRDKbx+NvHsCPn34bgL2BDbWmIYU7ZOIqMYqsSPg3/X2pNNXs1lVYM5OxA+HX/7BJ8xfWEuOFdTSV\n0wW6aomZr1mQyubx2q4hy9ftYlZlB+jv5gBFlJoDHsxqVyoaf/T0Nlv7F8dexFBG3a2KqllKoPDP\nTmn2oy3sw0gya3sGYjSdQ9jngcvFtEDbjr443u7VpwWa+ajLIbJKAGCDmle9/KantdczuQKS2bxu\n6EF3kx/9sXRJ+uTTW5RMMZ/b3MLdN5xEKlvQlaeLvv7GPikD8YzmJgGUvPeA16U754dMXCWCz921\nDmvUEWkXHz/TdD31pKJwM8YCjLE1jLHXGWNvMsb+dSIWVo6Ap7Jwy2lYRjwupp1IRndLMpPXfFuT\nweI2yy2+f/2emr+PWbzArBijFpgJ99xOxW1w1d3r8ZGfvlxi9VVL3MJNt9dgkY6mlFv3aS1BhHxu\nLJ3dam//qoCEDXnC5VwlImimvF8AnFunzBqJpXIlFwlAaaoks3c4iRXzO/D9jx2Lk+d1YF5XuOx+\nU5LFbRYyEhfvFtlV0qys/RWDe0M0m/vc++fpnr/slB6EfW5TF5xcqi4f6/2xTEl1acTv0VV27hlS\n/Peya+aJL5+m+51jZ7ZMeEYJYM/iTgM4k3N+LIDjAJzHGFte32WVx+ViCHhdOlfJT555W7eNfHsD\nFANUnHMMxDO44KhpOHVBZ4lVvW+keOKJhjbvZcTtYJN00n7n4c01f5+CieFnllZVC8xiH3HVB/qM\narWN19oXwrpoahOWz2vHxm+dA6B4sgvk9LRTF3Rq66iEOC7DBjEN+twI+9xa5pNMNKVUFQa9bq1N\nqfEOwIpoKqe75b/9smUASjOzDoykMK0liIuPn4kFUyLoL1MMlM0XkM1zzQf8wFWnAACmSkaV+B5k\ni3tOh3KRFZlfgqFEBhG/p6R/UU9HCPFMHlsOKHcHacOF+wtnzAcA7JY+i4FYWmdxA6pwS8ba9r4Y\n2sM+nRF4+BR9TrjR5z9RVBRuriDCy1713yFPtwj5PLrbVaOV2G5osSi2HUlmkckV0NXkR9hX2h5W\nWGJhn7uurpJ71uxCz6qHD7k7Rvj+bvqbo7H52+dpz9e6apRLh8wvPqOIQr2EW/btXnLibHz4uOkl\nn3OthPur5y3CPVeejOaAF20hr4mrJKuJUlvIZ1o5WG7/ZlZwp+pKMCL86Ywx9HQolrCdWY8AEE1n\ndcJ95qIpaAt5YUzIGUwUU/f2DCUxmspZutZEOqKISS2d3YZTF3TiwGhKM6TEMSD7uGdLQVU5qDwY\nz5Sc1wAwu0MfhL14qd51cbIaGD7j+89h73AShQLHYDyj83EDimUtX4h2DSZ0azHjUNV62PJxM8bc\njLENAHoBPMk5H1/HnBoQ9LqRzCjiks7l4XYxXL6iR3vdOChUWGGieGFeV1jt6623gMQX0dMZrmtW\nyc/UYIrRQptoRJBrWksQQZ8bIoOs1u0E5LmhYtRUvVxRcubCjRcdiaaAF9FUVpcWON5+1eJ4Ckn9\nKWa2hbBXEm7OlTS05qAiiC0hL4YSWVsVh6ICMmIS+OqKmA8JkK3m6a3KZ/zAa/bcXrFUTkutE7SG\nfBiWLnCpbB6ZXEG7gxBl7GZpg0Cxb4rcw0P0bBHBROEuaw7KFnfR/SIXLQ3GM1r5uUy7ZDlH/J4S\nIZfT9VZ87xmMprLIFTg6InqLuz3swzqpZev+kZT2Ocq8+a/n4sXrzgAAnHvklJLXJwJbws05z3PO\njwMwE8CJjLGjjNswxq5kjK1ljK3t6+sr3UmNCfrcSGaVg3vT3lHkCxzHzSr6DyMBD756XjFAKU40\nESBZOLUZYb9bO0EEwr89qy00bmuYc47frt5luh+36herNKUDUKzff3tks21/ZTWI9xdBrx9fshQA\nat42V/i4v/WhI7SL6midgpNCMB750qnwuF2IBDyIpXO6bKL/eGwL1o8jSBk3cWXMaA3qLsSJTB75\nAtesydagD5lcQcsYKYdIb4v4zdsam/u4i9a9yESxW4oQTeVKLhKtIa8ut3/EUCZ+yUmzAVgX+iRN\nLm73f/5kAMVYwKCWT60X5Js/fiwAfQ5+XzSNLpOuh9OlmZNf/8DiktdF4yrBgNTUSmbXYAKZfAH7\nVXfpQCyj828Lwn4PZrWH8Po3z9HOl4mmqqwSzvkwgOcAnGfy2q2c82Wc82VdXV01Wp41Qa9bE+P/\nevItAMCiqc3a600BDz5/+mH4+aXHAyieaMJ/O70lYDpJRwj3zLbguC3CF7f144YH38C/P7ql5DUR\nNDErpDCyZucgbn1hB2544I1xrccMIdziABW3ooM21lUN0VQOpy7oxGUr5moWaL2GYQghWTBFyX2O\n+D3I5rmuR/WO/jg++tOXx1ycIy74snDPbFOmsQiL2ui/FWlqdtwlZhcGQWeTz8JVUrTuAcWnniqT\nYVPyu0bhDnp1bidxoRUXIiG2Vsew+IzkPOels9sQ8rm1BAPREtVoSX906Uy8//AuLOhWvkPOObYc\niJq6uLqbA5qRJr5zmZaQF/f948mY2RaEx8U095HRx3216gvfeiCKfIFjNJXVZaiY7bdcn+56Yier\npIsx1qo+DgJYCaBUiSaYoM+tXdHFCbFQaiYj2lqKq30ym9dZvowx7YSWU9WGkxk0BTxoDXmRzhXK\nppZVQpQHm6Ucimo6Oz6ybJ5rf0Ot6Y+lEfF7tNtZUZq8p8p83UpEU0VrsDnghdvF6pJ2CCi39B4X\n044BIUgHTTJJvvBb8yGxlRAutrBkTc5oCyKVLWgXCBE/MLoWhm24ocTdgdF9AQBdkQCGEvpUv2y+\ngHXvDumLTwJe25Wa0VS25L26mwI4MFI6XUYT7jIZLsrzwrItCiRjDFNbAjgwqhxfb6tuFrMhBB1h\nn2acic9sl0Wl5udOm4dHrzkVx88x74m9rKcd1648HLkCx1OblTavRn+5KJLa3hdHNJUF58XvzGnY\nuVxMA/AsY2wjgFeh+Lgfqu+yKhPyuTUhe2jj/jLbKSdtPJ3DU2pf3lXnK1dncdLJfu7hRBatIa8W\nFBqPu0T406e0mKQmqq6SfTai0uKEMavyfHFb37h6KPdF9SXEs9QJKnttZiPYRfa/ulwM7WFfiTsm\nlc3XJNiTyhZ0mQfCav2U6i/96JIZ2muPbqrcb8OMhKnFrfhWxWcnuvqJbId2KahXiaFEFl43010Y\nBJ1NpZbuXa8ohVPPbS26KcN+t604TSZXQDpX0GUWAcpFfCCe1oyX4h2ER/e/GN1V8jeoFzCjQE5r\nCeDASAoDMWXfVn7isL+YgCCSD76hTrYx4nIxLJ7WbPqaQJzTd69WUhyNrpL2sA+tIS+298W0C4Vc\nzOMk7GSVbOScL+GcH8M5P4pz/u2JWFgllOBkvmJwL6z6CJOZPDKqhXLOEcqBIgpxZKtkOJFBa9CH\niOiBMI7beeEr+9lz20tei6onwf6Ryiex+BvNij4uvW0Nvv3QX8d8y29sk+lxu0rSompB1JAn3BH2\naRaZYNE3HsOy7zw1bks8mc3rhPvI6S261/92+Wz8+ooTx/UewgctuwHE2LBr7lGqNN+3QJmV+jdL\nlQvFAjWVzMpqlBmKZ9Aa8pnmCJvlcotj+DMn92jPRfxeW4aHVuxjcJWIXHBR2FMUM0XwxNruXbvb\nNOD68vZ+3faCqc1BHBhJ4d8fU27c37XIfHEx5QLWH0trLs6pZeozKmHM0DG6ZxhjOKwrgu29MS0o\n27DC7VSCPsXHvWW/Ys1cf755pWTIq4pzJq+5VsRtmfgi5QDlcFJvcUfHkctt1ZoyX+Ca1WVnNqAY\nALt+17ClQI/VUlUsbr2vrzngqWmqXi6vVMfJt+KdEb/WQB/Q5wsvufHJcbmo0tk8At7ioS38pIJZ\n7SGcdvj44jCJdA4hnxsuqZHTQlWYd6pCJPy4IqdaWKh2jAGl8ZK5aJiVvefU4+Kfz12oPSeCsmbt\nWWWs3DLT1HWLu0Lh72438fuaVVT+fq1iJRvdINNaAjgYTWv9wr9qce7+Sr2L+JVUet/dNA7hli5M\n377oSNMpQV0RP1a/M4gPqw3qWoITN7m9GhpXuL2Kq2S3ao3+jVp2+tQ/naY1UAeAkGZx50qKGsIm\nFvdIQql0EyfZeCxPMX7J2IRhUTBTAAAgAElEQVRm33BSs/7tpKXdt66Y0mXWP0LscywcGEmVVJk2\nB701LUcXn7ucJ9wbTWH9rmHtZ2Og8v9et9fTw4xkNq/7zF2GLnldqvBdc9YCAGPrHhjP5EsChy4X\nw6kLOnGsmt00msoi7HNrGR4etwtBr7vsIATBUMI6MGZW9j6aUvKw3dLf2mRimJghjnFjz40eNa3u\nnf6YuqYM3C6m2050Eix35+g2fP5TWgLIF7h2gTtjYbfp75135FQAwI+l4jqz9Dy7yBa3fGciM2jo\n72J18TzUNKxwh9Tg5K7BBEI+txbhnt/dhMO6IrrtAOVES2RycEvz6kSqVUz2casWtzhpxtrqNJ3L\naylPyWxeZ1HuHFAs6HmdYbzdGysr3iJwKlKe/u0R86rGantIAIpPOZ7Jl7SlVHoO1064zYThLdUv\nKixrY1rkePKsUwZXCVBsiwoUb/HFesbyt8bTOVP/c3dTQOslHU1ldfnJ4j3tGAND8YypZQsU85J1\nwp3MldQuFA2T8kFtK+Ge2RaCz+PS7vgG41m0hby6C6FwDxndHZxz+DwuXHmavjwdgGlKnxnfvuhI\n3c/rvr5S13CrWuy4PX7wsWN1P5drnXEoaVjhDvo8isU9mMTMtqBlv4CAxw3GlJzaeFrpmyC2FYHL\nhGoRFgpc83GLL3mshSjPbNaPTntj74j2+FcvK7eAl5yo5MFukl4zIvyKV50xHy1BL06aq4+ai3No\nLBa3OGGN4tJSY4t7VLsVLwrD9z56NIBiLw1RxLFy8RS4mD0/sBWpbKHkLmdeVwR3XHaClkcMFF1Z\nv3plZ9XvkcjkTOcNdjf7sX8khXQubyqmTar7ohLKrENzoQl4ld7WRovb+D0K10Cl1g3GoQICt4sp\n/afV9Q4b+mYDQE9HGAGvC09tPqjLchmIZ5DJFXQ51gK58MXKxQkoaX7zVTdXc8BjWjVZDWZVqEbE\nhQhQXFxm6ZhOoHGFWz0xlYkf1o3MXWrvhkQ6h3haHyAzZo7EMjkUuHJlbtJcJWMTMBH4/ITa1P4H\nTyi55pwX05FOUEW4XF/xQSkyf8phHTq/Oedcuw39zsObcdkda0z3YYUxS0DQHKitxR3TLLqiMEzX\n/KfKBWe9WrF27coFmN8dsVWYZEUym4ffW3pon7GoW5cudsHR0wAUfdDVEE/nTYVgizoKb+HXH9Pc\nFzIcSg75s1t78ccNe033zblqQJTJIZ7SHNA1yjJOfgGKrpJoKodnthy0vBgbhwrIhKS2EIPxUuH2\nuF2Y3hrEI28cwL/86U3t+Y17FDeYsToR0AtoT2f5JlVCSNvD5oHaavDbyLlmjGH9N87GlhvPw59X\nnTmu96snDSvcwgXSG02VWDWl23oQz+QRz+R0V1Cjj3s4XsxTDasTncfq435LTQETmQViqKgsvCJH\nVFSAmiH3BJ7SHNBVT6ZzBS3HG1BSwaq50AhLuJzF3RtNoWfVw/j92t2292vE7FZcE27VNypOqsXT\nmtERNq8MtIuZq8SMKepE8dyYfNw5LX4i880PFW/vX94+UPLZ7uiLY/P+UVx+x6u45p7SsXmA0mI1\nV+Bl/avTWgLYLx0LSvGNucW9ae8IrrhzLW56ZDOGExn0rHoYd72yU9vueTWF0MzC725S7iCiqSxW\nvzOopSLKCDl9cH3xQiQ6FS40GdQrB4vnVRBucTHZabPnSjkYY7juvIW4//OnlN2uPeyzdfwcShpW\nuIXF3R/L6KrFzAj73Uhmcoin8zq/pEgVFC06RdpTV5MfLhcD5/rAiCCbL5Qd2AoAd6kDCZbOKY5G\nyxc4npZcKCGTPHIjYk3dzX50N/sRTeW0tECzzI83yrhdjJh1ZgOUAazxTB6pbB4nflfpnXzdfRtt\n79eIyMyRLa0ZhoyFV3cOwedxwe1i6Gzymw4KsEvKEJy0Qr5wc87x7kDpxBcr4mr/aiNzO8M4/6ip\n2s/GdRxhyDU26/NRbDBlLdzTW4I699hostS6L/YSUf6ue17drWUz3a4OJQCKx6HcT0bgYgxr3hnE\nP/x6rfazEeFzlwvExLFp5t7wuF3Y+b0PYPUNZ2kpklaIc/QD6t3ReLnq9Pk4fk7puMJGo3GFWxLg\nShZ30OtWLO603uL2uZWxR+JE6dX6duj9csahxF+9fyNW3vw8/lQm80H0XZ7RGsS3PqQUDYwks9pB\n/oerV2julGQZV4ncS6RNC5gqJ4Wwxn1SwEZkstjhAdVCMt5ii1mFTxv89MbPwS5mrhLRnlQUkewe\nSmgB3KnN/nH1y1YKcCof2kJ4Y+kcbv/zTrz/P5/Dc1t7K/yWgoiXmHFCT9EdY8zoENPEBWf94Hn0\nrHpYFwQXvUyCPuu/oSPiQ180rQlkLJ0rKaARo8vkGMrPX9gBALpZkbkCt+yrfbw6k/UvO5TUPTOD\n5fuGgB5QDPiWmw5jJ/An3JinzO+ouO1komGFWz5pjLeIRsJ+D5IZJYNCDigxxnT9SkQ2gLiFFgif\ntEAI3p0mw2EFB0fTWlqYSPQfjKc1azni9yDoVQKn5XKmD46mEfS6EfF7NAtKWMpC9DJSUKiabIxt\nBxV3zrwufZ6zuChe/dv1WuoZYL+3s5FRi6yFDqlZEufAOWr6V0twfO0GUjl7Frdb7eseT+dw40N/\nBQBcb6MfDOccgwnzFqOA3vI8zPDZGsVVcLk0BEBcyAMmFrBAGCDHfOsJpHN5jCSzui55ALTb/bVS\nxzuRZincQxf95CU8tHG/ZW75BUfpLd1bPn18yTYz20K48NjpWvogoBgXPrfL1IqvhvfNV1yNcgM5\nooGFWz4xK0WLQz6lC6ASnNQfSBG/R0sH7I2m4XExzbIVEW+G4u2hbHWWm+58MJrCDDXnVASZRpJZ\nXcc0t4thRmvQsnIMULIuprYEwBjTov7C4jbreVHNnEGv24XTF3aV5Nl+4oRZ2uNoKqddgD52y8u4\n91X9RBQ7RFM5eN2sJDg0tSWAdwfi4JyjL5rWAlERk/x6u+TyhZKBseWQv38AWkZIOaLpHDK5gmVQ\n/PIVPehu8uMnn1qCGy7Qd6vrlI4ZOYNhXmdR4FPq+wcsLHoA+OwpPdpjcZc1taW6aeOFAsfr6uBb\nq2KxqYasEKtgYkvQi50DCc2S/+VL7+gMirHykSUzsO7rK0uqXyc7jSvc0kEdLHOAA8Wc70Qmp5s3\nKV4TArF7KImpLQEtT/Wi45RSZbmb22Y1awCwbsbPOdcVtsg9UYQ/Xdymz+uKYEe/tb/8oY37tZao\nYryTeF8RiHziy6dpFkk1Frfc+EnG73HjX1T3TjKbxyLVDzmUyOKr979RtaCKBkbGrIDFU5uwvS+u\npI7lC1o5s9ZuYAzCHUvnlOZAZTIyZMJ+j66CEwD+59nSFgUyotm+WaAOUILha762Eh88ZnrJRfHo\nGUUBkv3F8mYJ9UISKnPXIBsr/7dRsaKntZRmx3zwGGvfsJ3Au9zPQ0ywMUPULPz9r9ZW3Gc1MMZM\nM1MmO+8J4bbyNQrCPiV31iyFK+z3aH7IvmhKa/IPSG04Vcv2uvtex0d++jIApbxZRM6N9MXSSGTy\n2hQSLV88k0NStLpU1zynPYTdg+YuCOFCERZ50XLP6F6f1hLAH65egcO6wlUJ96hhXJXMrLbiba+x\nVWa1xTGxtPn7zOuKIJbO4c19ysVQE24pja1axO+YDSAwI+zz4FV16KvgQIX+MaLHiln6XCXkLAsx\ngcXrZrqBBcJ9ZFew1u1UXCFmVYXiczdLhZMND7lUXka+2B5Vxuo9Sr0gbT1YHC68crF5RSQxfhpW\nuEUPEgAVU3eGEhnsGUoimS0NKCm3ysrJPhTXlxkHvG4EvC7tABe9FwDlBLSyuMVUD+EDFSfPwdE0\nvq/mc4s+vi1BL0ZT5lNRhA9bnFRthqKgaCoHFyta7x1hv21R5ZybtvIUyLfExkyCatwxYp1mwi0u\njCJ1slsV7vHk0Gvl9TYLJ8J+t/Z5PnjVKVgyu7WiL19Y6OXqB6wI+ZQ+8f987kL85FNL8ZWzD8eS\n2W26Y0kEpMu54oBiNaioxO02CfaJ7zedK+B3/6AfFStSMVedv0jrRV2Ocr2nr1XbBwDFTBk5SEvU\nloYV7oAUca9kcT8rtbo0WtytIS+G4hkkMjlsPRjVWdyAMrXErOy9PeyzDCqKSjNh9YkT/HdrSv3D\nYb8HnMN0Kop4X5FCFvS64XO7tDuA0WQWEb9Hc+20h322hTuVVXLArVIpZ7UXb7s37R3Rzd4biGfw\n57f7cfMTW229VzSVNY1DiKCyaNkpAqFaKfoYLG6rTndWiEBiS9CLJbPbEPS68fL2gbJ+bmERW7lK\nKvHV8xShnNEaxBfPWoBWQ4uB/lgaPrerJNvHiPA/b++LI+B1mV6s5IyjpXP0AT5Ryt5V4QL0i88s\nw3c+XDL0SofLxXDp8jkAgLXqHUyleY3E2GlY4W6VunZVyiC49dJiJNzoZ1W61GVwwX+/CECfJgWo\nwp7IlgzPbQl6EU3lTBsUaelv6okU9LnREvSaFnoUe6mUipQQaOHbZoyhJeSVXCX6oouOiM9Wt0Gg\nfKN+QJ/Tu+r8RXjiy6fh4S+9D4AyHedvf7kaP3rmbVsNmqIm8wyBog9X9KsWFniLIXumGsRnb6e8\nGSj6k8VkcZEWt3GPdT58XywDxsy75I2FlqBXF2juiyqtditVCooLOQDL7bNSK2PxnYrPWTSPMg7N\nNXL2EVPwaVWUy3HhcdMBAJvVjp3GwCZROxpWuOXbtkqukmXSLZtxokVXkx/RdE6rzBIHn6Al6MVI\nIlvSNUwIplkaVdzE6pvZFsQO9RbyDrWjGiBN6DHJ5RZBUbnMuC+axu/W7MaFP3kJ//f6Pt1tfUfE\nj6FExpaYalWTZfNsFUusq8mPgNetDQqQXSV2AohWrpLjZustQCG2ZsJdKHD87LntFZt+ibudcvnD\nMqLsXUje5SvmAgB+oxZQmbFh9zA4x7gaHsm0hvS9Yfpi6ZIm/2YwxrRqRyurWYjnCjWt7s1/PRd/\nvHoFgKLFPRZfvRnClSdcJWNxJRH2aFjhlqnkKpFLh080NGkyniDGgy2iBi/lQOT0loAmeMPJTIk1\nrrlKJKtvOJGF0FN5eKlWvWdice9Rg5Zm45M27hkpseA7wj5wbs8HLdwQ5YqX/nL9Wdjxbxdollxz\nwKM2ty/u344fOprKmt7G+z1uXCaltRW79pUK9/Nv9eHfH9uCr6tTZaywU3UoI7I5jlSDayJF748b\nrIurXnirtsOwW4JeJLN5zT3TbzGk1gxxUbfqU33p8jn4zoePwqfUwb5hv0cLer7Trwq3zW59lRDx\nIVGkQ8JdP94Twl0pHZAxplUyGg9ScXCJ20eR8C8I+T1IZPKahXrLp4/H0185XbMKr/z1Osz/2qO6\n3ym6SorisVcqT5bvFsqVvf9AHYIs5yR/ztAmUx7lJP42O30+hOVarniJMaZr4ckYQ9jn0V3EhuLl\nhZtzrmaVmL+PWQm16Pk8mlRcVIUC10r5y42pA4oZL3bzuEVAT6Tp+T1K5z0r0cnWIDfZSIuU5w8U\nXSV2EHc8Vi0CPG4XPr18jm5oQFitIdihCvd4u+4JhIGxfyQFn8dV8bwkxs57QrjlDBMrbr/sBPz+\ncyeXtOIUlspwIotzjphS4icMed1IZHJa8Ki72Y+gz60Jnkh/krNCigMbigeuPHFljlRhJtZTruxd\nFs/rL1iMbd89X/v5MKlU2a2u/acV8pABaENgq21MH/S5dWXPWw6Mltla6XxY4NauC9GA32dwO7SG\nvBhOZDD/a4/isjtfrRioEwzFMwj53LZF45ITZ+PHlyzRujgC5XOfhU98+bzaZUxorqFEFsOJDPpj\nadOcbDOEq+z9VUz0kYu5mvyemjVU8rhd2vdcqQ0FMT6c2Wy2SsxaeBppDflK3CSAIsSC6SbtPUN+\nNxLpfIlrwWjRpXPFAbVDiYxu8gmgBEjX7xrCCT3tOuunXHASMO+eJv++3JJ0+bwO07WZIZpXVXs7\nG/Z7sK23mKtbadhx1KRPicx/f3IJjnh+O/7x9MN0z7eHfBhUA3YvvNWHE3uKjYGy+YLp2ClAHYRR\nxWRut4vhQ8fq4xrtYR+GEhkUCrxkek5C7eQoirNqgdzKQOSIGzNAKvEFG+l8Ms0BDwbjmZq5SQRt\nIR+iqRxaKjR+I8ZHQ1vc//aRo9EW8trqs2tFh3SbaOw1DKjZI+kcBtVbUW3CtUEc5GrCfcPJkotA\nwOvGKYd1lgiO8HF/7q51un1wzuH3uLDyCPMJ2Deq6VkzpUKZtrAPTaofuhJ90TTawz5LAbQi6HXr\nhvyKC4AVwgdulZ7n87jwxbMWlKyjI+LHzv5itz65GKdcr+7hRFZzPYyVjrAf+QLH+l1DJa8Jl1al\nuEo1yMFYcQcif6/l+Pmlx+PK0+aVXGDsvmet3CQCEU+q1D+IGB8NLdyfOmk2XvvmOeNqsO5xuzSf\nc9ikv7I4wPep3erEAWm8dd8ptQQ9MJq2nQolC4A8T7IvmkY6VyjJKxdcunwOdn7vAyUuga6IH/02\nUgJHUznbfmAZ+TPye1wV88atGkxVojPi003BkS8W5YR7JJmpyuI2Q/yNF9/ySslrokmYWUvXsSL3\noBHZTXYt4XOPnFrSD8UO4jiudTl5l+p6HMuxRdinoYW7VoiMB7NRVCIzZO9QUu125tI9L/i/14tB\ns4MjKa18uxKycMul12IIcqUJIUY6Ir6S3htmxNUp5dUiV5bO7Qxr1Z1WCH+/XR+1wCgookgHqGxx\n25ktWI7TLYbXAsVpRWZDFMZKcUxeBj96ehsA+5WfY0UIt520w2oQKaQk3PWFhBvF23gzIRO+2T3D\nSTQHPZp1b7TyhXWYL3D0xaqxuIsnqJwN0hdVBLFSVZsRZXpMZYs7ZuhNbhf5r57dHkJ/vPxFQnOV\n2EzPE3SY3MKLzCBjTr1MuenodpH7RBtz4uthcTcHvPC4mK54arxjuiq/Z7FNQi0R7sZq77CI6iDh\nRtF6NstEEAf467uHLSPlzQGPlsrVH0sjX+CmfSPMcLsYbvzwUfB7XBhNKpNYXts1pGVrVBs87Gzy\naReAS29bjZ5VD5vmWhvnb9pln9SAaXprEL2jadM+KwKrCeKVMPu7z1qs+PutinA454qrZJwWt4yx\nerMePm6Xi6Ej4sMu1U1yxkL7GSJjRXxltfZxi8+eob4XnskOCTeKomKWFiVnQzRZ3P71dIa1E1xM\nbrHrKgEUf/UnTpiF0VQWF9/yCj7y05fxw6eUW+ZqT6zupgCGE1lc/LOX8eK2fgDApr2lKXvGaUB2\nke8AZrQGEUvnLLskAnJpfXXvJVu9/++cwwEAf/e+ufC4mNYUykgik0c2z8ft45YxXiSExW1sDzxe\nOiN+PPyG4m4bwwjMqhGzSs3iOuNBBJkLZS7mxPgh4UbxNt4sl1oeoGo14qunI6zleR8YrV64AeUW\ncySZxTppWklryFu2I5sZIpgpTz0x9vveP5LEzoEEXn1H387UDsfNUtLyHrzqFK0RlfDHmxFL5cBY\n9a4F0TPk5HkduPqM+XjnpgsQ8LrRGjJv+gVAa41aC//qbZ9dptunIKH1U6+t4Ml3GKIXej0RDbLG\nO6HGiMhumYiLz2SGhBvA2UcowSh5IolAzpM2pjj94eoV+OKZ89HTGUZ/LI1UNq9NYZ9S5TSSaS0B\nGI2UuVUGJgHzbATRk0Lwqtq/+cBo9XMdv3DmfNz/+VOwZHablrJmbMwlE03nEPF5qk5Xm9IcwIvX\nnYG7//4kMMY0n6/f48KfNuwzHeYg2unWwlUi7nR6DZ+RFpysoY8b0Av3WFxY1XLp8jn46JIZONsi\n3XSsTFcNB7m7JFF7SLgBfOKE2Vh9w1laM3gZxhgWq21VUwaL/KgZLfjKOQuxoDuCAgcWfeMx3PXK\nu/C4GDqrDPrIed+iHeZYus8ZeyC3hrxacyuBGObwyJdOrXr/bhfTpmSLCtB3ykxHT6TzY87AmNUe\nKhH8vcNJxDN53P5S6bxPbWp9DSxuUbn4j79Zjz+/3a89H0/n4HGxqu+EKiG3iB2LC6taZraFcPMn\njqv5e525qBu3X7YMV546r/LGxJgh4VYpN3H6ZLUiUe43IiMPMt3WG0N3k79qC1OeMiPKl8ciDk0B\nL9656QLt51MO68CzW/t0QTbRQlQuvR8LTQEvprUEsO2g9ei1ZNbe4F67XHeeMlRip8mczpEaukrk\nXiEiVgCMPTZQ8f0ki7uWn9dEwxjDmYum1KxzImEOfbo2+Or5C7FoahOuXXm46euzDA3j7WaUyMg+\n8WNmKpb/WK06OZVMjE+74L9f1Bplidv9WgjE4VOadHM4jSSzeQRr6Fa46vT5WDyt2dTPLfqU10K4\n5VmRHHIfmtLxd7VAdpVUe9EnJh8k3Dbwe9x47NrTcN5RUy23eeYr78c5qr9wLHEZxhh+funxuP/z\np2i+bo9r/F+PaOe5dziJL/3uNQBKZkTQ666JQCyf14EtB6KWzaZS2TyCNnrJVEN3k990KnktLW4A\n+OEnjgOg78cSS2frkqNMLVCJaqh4RjHGZjHGnmWMbWaMvckYu2YiFtZozOuK4ANqV7mxCse5R07F\n8XPatD7btbjbnNkW0gJFz6kj3BKZ0tmbY+UktUveHouBx4lMvubtPRXhLg2sxtL5MWWwWPHhJTNw\nymEd2CtlzcTT+bq4SsY6Bo2YnNiRhhyAr3DOFwNYDuBqxlj985UakJyaGzvePOKjZijB0DMXjX1K\n9surzsQL/3wGAGD5XMVH73UrFnYiM/aAoRHhNrjp0c2mrycztfVxA0pHx/6Y0r1PJpHOIVSjOwnB\njNagbspQdIyFS5Ugi5uohorCzTnfzzlfrz6OAtgMoHY9Ld9DiMkwbeNMRztmZive+NY5OO8o677Q\nlZjeGsRsNfh444ePwuJpzVoKWzyds9XD3A6ij/b2PvPMkpFktuad4qa2BJEvcBw0WN3xTL7mhTEz\n2oLojaa1Abgxi8HH40WUii+b01ZhS4Ko0sfNGOsBsATA6nosptH54DHTMa8rrM0tHA9W/avHQsDr\nxsrF3RhNZVEocCSztbO45ZYapnMzExndYOdaMEXN+DA2uEpkcjUvjBFB44tveQXPbe3FYLy2JfUC\nt4vh0WtOxW2fPaHyxsSkx7ZwM8YiAO4HcC3nvCQSxRi7kjG2ljG2tq+vtjP5GoWpLQE885XTq+7o\nNxG0BL3gHIhlcmPuDGjGnI4wVi5WXDrGkWmZXAHxTH7cdyBGxEVttKSPSK7mhTFymuimvSMYSmS1\nPPtas3haM1rqcFEg3nvYEm7GmBeKaN/NOX/AbBvO+a2c82Wc82VdXfVvkkNUR7M0HksJTtZO4C4+\nXhn7FTVMvBdT6ltr3MioWZ2uMmp4PyVwWFuLWx7S8f0nlBmgdoccEES9sJNVwgDcBmAz5/zm+i+J\nqAfN0uT00WS2pt3thM83ZihDF4U+tba4xd/y1sGo7vnRVLbmFvd8qTBKMLXKdgYEUWvsWNwrAFwK\n4EzG2Ab13wWVfolwFiJFcf9ICvtGUphTw9t90c88lta7LobU/tK19nELH/PNT76lPcc5x/a+mNac\nqlZ0NwWw83sfwLEzi+0Qls6mACJxaKlonnDOXwKouW6jI4R7zTsDAICjZ1Y3jLYcomuisXf1UA2b\nPsnIgdtUNo+A141EJo9UtlB1V0a7iIk8s9qDdR9yQBCVoMrJSYJoT/vrV94FACye1lSzfYtpP8aJ\n76Isva3GPm4A+Jba+lS4Z8Tsy3q8F1DsJdJU5SQfgqgHJNyTBDGiKp0rANC3qx0vIZ8HHWGfrlAF\nKPayrrWPGygGW2NqgFLk0I+lo6IdutVZirWMDRDEWCHhniTIDau+cvbhNb/dn9EWxOb9o7pBvkOJ\nDHweV1263Ql3ichkKVrc9bGIRVpgMluaq04QEw0J9yRi5eJuXHLiLHzxrAU13/fMtiA27B7GCd99\nSntuOJ5Fa9BbF5+wGBghepaIC0ZXpD4+7hnqkA2jH58gDgU0inkS8cs6VuV1NxUFM5rKoingxVAi\no5Vy15qZqqtH9EgX3QLlPtq1ZJYq3MZcdYI4FJDFTdQEuQOgyK8eTmTrUh4OKE2ZfB4X9qp+9b5o\nGk1+T807EQq61AtTR52CnwRRDSTcRE0QTZgAYPN+RbjX7BxEqk4+YZeLKZ37NIs7ha7m+hXGtAS9\nuPGiI3Hn5SfW7T0Iwi4k3ERNuOi4YsPILQdGtWBhrTsDysgtV3tH07rxX/Xg0pN7tI6LBHEoIeEm\nasKnl8/BlhvPw4k97fjjhn1aw6mLj59Zt/ec0RrUXCVDiYw2mZ0g3uuQcBM1I+B14yNLZyCayuE/\nH98KADXvHSIzoy2I/lgaqaxSNdnIQ3YJohpIuIma8vFlSqfA7X3K5Pd6FqyI9qrv9MeRzuXhJ+Em\nJgkk3ERNcbsY5nSENB93oI5ieuwspd/KPWt2IZUtIFDjocQE4VToSCdqTmvIp7V0raf7QnQ4fPzN\ng1qzKYKYDJBwEzUnmipWF9bTVeJyMczrCuPAaAq5AkfAQ8JNTA5IuImac8yMYu/qehXECD590hzt\nMblKiMkCHelEzbn548dpj+st3LOkgRDkKiEmCyTcRM1xuYpNpcJ1TAcEgJ4OWbjpcCYmB9RkiqgL\nP7pkCQ6MJOF21XdazOwOsriJyQcJN1EXLjx2+oS8j18KSPopOElMEujekmh4Fk5RxrD5PDQLkpgc\nkHATDc/d/3ASrlgxF8fPaT/USyGICYFcJUTD0xnx45vq8GCCmAyQxU0QBNFgkHATBEE0GCTcBEEQ\nDQYJN0EQRINBwk0QBNFgkHATBEE0GCTcBEEQDQYJN0EQRIPBOOe13yljfQDeHeOvdwLor+Fy6gGt\nsTbQGmtDI6wRaIx1Hso1zuGcd9nZsC7CPR4YY2s558sO9TrKQWusDbTG2tAIawQaY52NsEaAXCUE\nQRANBwk3QRBEg+FE4XZMBa4AAAUTSURBVL71UC/ABrTG2kBrrA2NsEagMdbZCGt0no+bIAiCKI8T\nLW6CIAiiDI4RbsbYeYyxrYyxtxljqyb4vW9njPUyxjZJz7Uzxp5kjG1T/29Tn2eMsR+p69zIGFsq\n/c5n1e23McY+W+M1zmKMPcsY28wYe5Mxdo3T1skYCzDG1jDGXlfX+K/q83MZY6vV97uXMeZTn/er\nP7+tvt4j7et69fmtjLFza7VGaf9uxthrjLGHHLzGnYyxNxhjGxhja9XnHPN9q/tuZYzdxxjboh6b\nJztpjYyxhernJ/6NMsauddIaxwTn/JD/A+AGsB3APAA+AK8DOGIC3/80AEsBbJKe+w8Aq9THqwD8\nu/r4AgCPAmAAlgNYrT7fDmCH+n+b+rithmucBmCp+rgJwFsAjnDSOtX3iqiPvQBWq+/9ewCfVJ+/\nBcDn1cdXAbhFffxJAPeqj49QjwE/gLnqseGu8Xf+TwB+C+Ah9WcnrnEngE7Dc475vtX9/wrA36uP\nfQBanbZGaa1uAAcAzHHqGm3/LYfqjQ0f6MkAHpd+vh7A9RO8hh7ohXsrgGnq42kAtqqPfw7gEuN2\nAC4B8HPped12dVjvHwGc7dR1AggBWA/gJCgFDR7jdw3gcQAnq4896nbM+P3L29VobTMBPA3gTAAP\nqe/pqDWq+9yJUuF2zPcNoBnAO1BjZU5co2Fd5wD4s5PXaPefU1wlMwDsln7eoz53KJnCOd8PAOr/\n3erzVmudsL9BvV1fAsWiddQ6VRfEBgC9AJ6EYokOc85zJu+nrUV9fQRAR73XCOCHAK4DUFB/7nDg\nGgGAA3iCMbaOMXal+pyTvu95APoA3KG6nX7JGAs7bI0ynwTwO/WxU9doC6cIt9l4bqemu1itdUL+\nBsZYBMD9AK7lnI+W29RiPXVdJ+c8zzk/DopVeyKAxWXeb8LXyBj7IIBezvk6+eky73cov+8VnPOl\nAM4HcDVj7LQy2x6KdXqguBh/xjlfAiAOxe1gxSH7LNWYxYUA/rfSphZrcZRGOUW49wCYJf08E8C+\nQ7QWwUHG2DQAUP/vVZ+3Wmvd/wbGmBeKaN/NOX/AqesEAM75MIDnoPgJWxljYjC1/H7aWtTXWwAM\n1nmNKwBcyBjbCeAeKO6SHzpsjQAAzvk+9f9eAA9CuRA66fveA2AP53y1+vN9UITcSWsUnA9gPef8\noPqzE9doG6cI96sAFqiRfR+UW5o/HeI1/QmAiBx/FopPWTz/GTX6vBzAiHqr9TiAcxhjbWqE+hz1\nuZrAGGMAbgOwmXN+sxPXyRjrYoy1qo+DAFYC2AzgWQAXW6xRrP1iAM9wxYH4JwCfVDM65gJYAGBN\nLdbIOb+ecz6Tc94D5Th7hnP+t05aIwAwxsKMsSbxGMr3tAkO+r455wcA7GaMLVSfOgvAX520RolL\nUHSTiLU4bY32OVTOdZPAwQVQMiW2A/jaBL/37wDsB5CFcmX9Oyh+zKcBbFP/b1e3ZQD+R13nGwCW\nSfu5AsDb6r/La7zG90G5NdsIYIP67wInrRPAMQBeU9e4CcA31efnQRG1t6HcqvrV5wPqz2+rr8+T\n9vU1de1bAZxfp+/9dBSzShy1RnU9r6v/3hTnhJO+b3XfxwFYq37nf4CSceG0NYYADABokZ5z1Bqr\n/UeVkwRBEA2GU1wlBEEQhE1IuAmCIBoMEm6CIIgGg4SbIAiiwSDhJgiCaDBIuAmCIBoMEm6CIIgG\ng4SbIAiiwfj/NqQXexHdn28AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f550d188e80>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "N = 100\n",
    "plt.plot(np.convolve(ef.history[:,'batches',:,'train_loss'][0], np.ones((N,))/N, mode='valid'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate performance on the training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_data = X[:10]\n",
    "pred = ef.predict(eval_data)\n",
    "pred_proba = ef.predict_proba(eval_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['???', '???', '???', '???', '???']\n",
      "['i', 'm', 'a', 'to', 'to', '.', '.', 'EOS', 'SOS', 'SOS']\n",
      "\n",
      "['???', '???', '???', '???', '???']\n",
      "['i', 'm', 'a', 'to', 'to', '.', '.', 'EOS', 'SOS', 'SOS']\n",
      "\n",
      "['???', '???', '???', '???']\n",
      "['i', 'm', 'a', 'to', 'to', '.', '.', 'EOS', 'SOS', 'SOS']\n",
      "\n",
      "['???', '???', '???', '???', '???']\n",
      "['i', 'm', 'a', 'to', 'to', '.', '.', 'EOS', 'SOS', 'SOS']\n",
      "\n",
      "['???', '???', '???', '???', '???']\n",
      "['i', 'm', 'a', 'to', 'to', '.', '.', 'EOS', 'SOS', 'SOS']\n",
      "\n",
      "['???', '???', '???', '???', '???', '???']\n",
      "['i', 'm', 'a', 'to', 'to', '.', '.', 'EOS', 'SOS', 'SOS']\n",
      "\n",
      "['???', '???', '???', '???', '???']\n",
      "['i', 'm', 'a', 'to', 'to', '.', '.', 'EOS', 'SOS', 'SOS']\n",
      "\n",
      "['???', '???', '???', '???', '???']\n",
      "['i', 'm', 'a', 'to', 'to', '.', '.', 'EOS', 'SOS', 'SOS']\n",
      "\n",
      "['???', '???', '???', '???', '???']\n",
      "['i', 'm', 'a', 'to', 'to', '.', '.', 'EOS', 'SOS', 'SOS']\n",
      "\n",
      "['???', '???', '???', '???', '???']\n",
      "['i', 'm', 'a', 'to', 'to', '.', '.', 'EOS', 'SOS', 'SOS']\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for (sent_in, sent_out) in zip(eval_data, pred):\n",
    "    print([input_lang.index2word.get(w[0], '???') for w in sent_in])\n",
    "    print([output_lang.index2word[w] for w in sent_out])\n",
    "    print()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
